{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f168fcee",
   "metadata": {},
   "source": [
    "[Python爬取二手房源数据，可视化分析二手房市场行情数据 - 知乎](https://zhuanlan.zhihu.com/p/415367897)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3244b1c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install requests\n",
    "# pip install parsel\n",
    "import requests # 数据请求模块 第三方模块 pip install requests\n",
    "import parsel # 数据解析模块\n",
    "import re\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "365578ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 发送请求, 对于房源列表页发送请求\n",
    "\n",
    "url = 'https://bj.lianjia.com/ershoufang/pg1/'\n",
    "# 需要携带上 请求头: 把python代码伪装成浏览器 对于服务器发送请求\n",
    "# User-Agent 浏览器的基本信息\n",
    "headers = {\n",
    "    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'\n",
    "}\n",
    "response = requests.get(url=url, headers=headers)\n",
    "# print(response.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8c552161",
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('/Users/msxr/develop/tmp/二手房数据.csv', mode='a', encoding='utf-8', newline='')\n",
    "csv_writer = csv.DictWriter(f, fieldnames=[\n",
    "    '标题',\n",
    "    '市区',\n",
    "    '小区',\n",
    "    '户型',\n",
    "    '朝向',\n",
    "    '楼层',\n",
    "    '装修情况',\n",
    "    '电梯',\n",
    "    '面积(㎡)',\n",
    "    '价格(万元)',\n",
    "    '年份',\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "57fadecc",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_text=''\n",
    "# 解析数据\n",
    "selector_1 = parsel.Selector(response.text)\n",
    "# 把获取到response.text 数据内容转成 selector 对象\n",
    "href = selector_1.css('div.leftContent li div.title a::attr(href)').getall()\n",
    "for link in href:\n",
    "    html_data = requests.get(url=link, headers=headers).text\n",
    "    selector = parsel.Selector(html_data)\n",
    "    # css选择器 语法\n",
    "    # try:\n",
    "    title = selector.css('.title h1::text').get() # 标题\n",
    "    area = selector.css('.areaName .info a:nth-child(1)::text').get()  # 区域\n",
    "    community_name = selector.css('.communityName .info::text').get()  # 小区\n",
    "    room = selector.css('.room .mainInfo::text').get()  # 户型\n",
    "    room_type = selector.css('.type .mainInfo::text').get()  # 朝向\n",
    "    height = selector.css('.room .subInfo::text').get().split('/')[-1]  # 楼层\n",
    "    # 中楼层/共5层 split('/') 进行字符串分割  ['中楼层', '共5层'] [-1]\n",
    "    # ['中楼层', '共5层'][-1] 列表索引位置取值 取列表中最后一个元素  共5层\n",
    "    # re.findall('共(\\d+)层', 共5层) >>>  [5][0] >>> 5\n",
    "    height = re.findall('共(\\d+)层', height)[0]\n",
    "    sub_info = selector.css('.type .subInfo::text').get().split('/')[-1]  # 装修\n",
    "    Elevator = selector.css('.content li:nth-child(12)::text').get()  # 电梯\n",
    "    # if Elevator == '暂无数据电梯' or Elevator == None:\n",
    "    #     Elevator = '无电梯'\n",
    "    house_area = selector.css('.content li:nth-child(3)::text').get().replace('㎡', '')  # 面积\n",
    "    price = selector.css('.price .total::text').get()  # 价格(万元)\n",
    "    date = selector.css('.area .subInfo::text').get().replace('年建', '')  # 年份\n",
    "    dit = {\n",
    "        '标题': title,\n",
    "        '市区': area,\n",
    "        '小区': community_name,\n",
    "        '户型': room,\n",
    "        '朝向': room_type,\n",
    "        '楼层': height,\n",
    "        '装修情况': sub_info,\n",
    "        '电梯': Elevator,\n",
    "        '面积(㎡)': house_area,\n",
    "        '价格(万元)': price,\n",
    "        '年份': date,\n",
    "    }\n",
    "    csv_writer.writerow(dit)\n",
    "    my_text=my_text+'\\t'.join(\n",
    "        [title, area, community_name, room, room_type, height, sub_info, Elevator, house_area, price, date]\n",
    "    )+'\\n'\n",
    "#     print(title, area, community_name, room, room_type, height, sub_info, Elevator, house_area, price, date,\n",
    "#           sep='|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e4276996",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "csv_writer.writeheader()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "aff23301",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/msxr/develop/tmp/二手房数据.txt', mode='w', encoding='utf-8') as fw:\n",
    "    fw.write(my_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3674ce42",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
